﻿using System;
using System.Collections.Generic;
using System.Net.Http;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using HtmlAgilityPack;
/*编写爬虫程序，实现以下功能：
1）使用多线程、并行编程或异步编程技术实现爬虫程序；
2）接收用户输入的关键字，用户点击搜索时使用搜索引擎搜索关键字；
3）从搜索的网页中寻找100个有效的、不重复的高校电话号码（使用正则表达式），并记录每个电话号码所属的url（可能有多个）；
4）显示已爬取的URL。
*/
namespace WebCrawler
{
    class Program
    {
        static async Task Main(string[] args)
        {
            Console.WriteLine("请输入搜索关键字:");
            string keyword = Console.ReadLine();

            List<string> urls = await Search(keyword);
            List<string> phoneNumbers=await ExtractPhoneNumbers(urls);
            foreach (var url in urls)
            {
                Console.WriteLine($"已爬取的URL: {url}");
            }
            foreach (var phonenumber in phoneNumbers)
            {
                Console.WriteLine($"电话: {phonenumber}");
            }
        }

        static async Task<List<string>> Search(string keyword)
        {
            List<string> urls = new List<string>();
            string googleSearchUrl = $"https://www.google.com/search?q={keyword}";

            using (HttpClient client = new HttpClient())
            {
                string html = await client.GetStringAsync(googleSearchUrl);

                HtmlDocument document = new HtmlDocument();
                document.LoadHtml(html);

                var nodes = document.DocumentNode.SelectNodes("//a[@href]");
                if (nodes != null)
                {
                    foreach (var node in nodes)
                    {
                        string url = node.Attributes["href"].Value;
                        if (url.StartsWith("/url?q="))
                        {
                            url = url.Substring(7); // Remove "/url?q="
                            int index = url.IndexOf('&');
                            if (index != -1)
                            {
                                url = url.Substring(0, index); // Remove additional parameters
                            }
                            urls.Add(url);
                        }
                    }
                }
            }

            return urls;
        }
        static async Task<List<string>> ExtractPhoneNumbers(List<string> urls)
        {
            List<string> phoneNumbers = new List<string>();

            foreach (var url in urls)
            {
                using (HttpClient client = new HttpClient())
                {
                    string html = await client.GetStringAsync(url);

                    // Use regular expression to find phone numbers
                    var matches = Regex.Matches(html, @"\b(\d{3}[-\.\s]??\d{3}[-\.\s]??\d{4})\b");
                    foreach (Match match in matches)
                    {
                        phoneNumbers.Add(match.Value);
                    }
                }
            }

            return phoneNumbers;
        }
    }
}
